{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on package selenium:\n",
      "\n",
      "NAME\n",
      "    selenium\n",
      "\n",
      "DESCRIPTION\n",
      "    # Licensed to the Software Freedom Conservancy (SFC) under one\n",
      "    # or more contributor license agreements.  See the NOTICE file\n",
      "    # distributed with this work for additional information\n",
      "    # regarding copyright ownership.  The SFC licenses this file\n",
      "    # to you under the Apache License, Version 2.0 (the\n",
      "    # \"License\"); you may not use this file except in compliance\n",
      "    # with the License.  You may obtain a copy of the License at\n",
      "    #\n",
      "    #   http://www.apache.org/licenses/LICENSE-2.0\n",
      "    #\n",
      "    # Unless required by applicable law or agreed to in writing,\n",
      "    # software distributed under the License is distributed on an\n",
      "    # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n",
      "    # KIND, either express or implied.  See the License for the\n",
      "    # specific language governing permissions and limitations\n",
      "    # under the License.\n",
      "\n",
      "PACKAGE CONTENTS\n",
      "    common (package)\n",
      "    webdriver (package)\n",
      "\n",
      "VERSION\n",
      "    3.141.0\n",
      "\n",
      "FILE\n",
      "    c:\\users\\92519\\anaconda3\\lib\\site-packages\\selenium\\__init__.py\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import selenium\n",
    "help(selenium)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调用模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time\n",
    "from random import random\n",
    "import requests\n",
    "import os\n",
    "import base64\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageEnhance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "from selenium import webdriver\n",
    "import time\n",
    "\n",
    "wd = webdriver.Chrome()\n",
    "wd.get(\"https://www.baidu.com\")    # 打开百度浏览器\n",
    "wd.find_element_by_id(\"kw\").send_keys(\"selenium\")   # 定位输入框并输入关键字\n",
    "wd.find_element_by_id(\"su\").click()   #点击[百度一下]搜索\n",
    "time.sleep(3)   #等待3秒\n",
    "wd.quit()   #关闭浏览器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 打开浏览器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-488-783dcc5c7ea8>:19: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "\n",
    "\n",
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\n",
    "# opts.binary_location = \"C:\\portable\\PortableApps\\IronPortable\\App\\Iron\\chrome.exe\"\n",
    "# opts.binary_location = \"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "\n",
    "# \"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 打开知网并进行自动化爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://www.cnki.net/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击高级检索\n",
    "element = driver.find_element_by_xpath('//*[@id=\"highSearch\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CDwindow-DD5B611BFFB5A68EEDC1FCDE2EBC60F9'"
      ]
     },
     "execution_count": 491,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#检查窗口位置\n",
    "driver.current_window_handle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 492,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 切换到新页面 \n",
    "driver.switch_to.window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择期刊\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[1]/div/ul[1]/li[1]/a/span')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 494,
   "metadata": {},
   "outputs": [],
   "source": [
    "#勾选来源类别\n",
    "def laiyuan():\n",
    "    element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[4]/input')\n",
    "    element.click()\n",
    "    element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[3]/input')\n",
    "    element.click()\n",
    "    element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[2]/input')\n",
    "    element.click()\n",
    "    element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input')\n",
    "    element.click()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点专业检索\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/ul/li[4]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AI_新媒体_query = '(SU=\"新媒体\" and SU=\"人工智能\") OR (SU=\"AI\" and SU=\"新媒体\")'\n",
    "AI_新媒体_query = '(SU=\"新媒体\")'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/textarea')\n",
    "element.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "metadata": {},
   "outputs": [],
   "source": [
    "#搜索框填入信息\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/textarea').send_keys(AI_新媒体_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 499,
   "metadata": {},
   "outputs": [],
   "source": [
    "#勾选来源类别\n",
    "laiyuan()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {},
   "outputs": [],
   "source": [
    "#50\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[2]/div/div/div/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[2]/div/div/ul/li[3]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 503,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>篇名</th>\n",
       "      <th>作者</th>\n",
       "      <th>刊名</th>\n",
       "      <th>发表时间</th>\n",
       "      <th>被引</th>\n",
       "      <th>下载</th>\n",
       "      <th>操作</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>新媒体文学批评在“超真实”世界中的时代变构</td>\n",
       "      <td>史丽娜</td>\n",
       "      <td>当代文坛</td>\n",
       "      <td>2021-06-29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>北京市医院短视频平台传播力现状及展望</td>\n",
       "      <td>韩扬阳; 李与涵; 高天; 郭蕊</td>\n",
       "      <td>中国医院</td>\n",
       "      <td>2021-06-28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>新媒体视野下常德丝弦的创新与传播研究  网络首发</td>\n",
       "      <td>田正铁</td>\n",
       "      <td>四川戏剧</td>\n",
       "      <td>2021-06-25 14:42</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>“在场”的言说：新媒体时代影视评论刍议</td>\n",
       "      <td>梁振华; 何庆平</td>\n",
       "      <td>中国文艺评论</td>\n",
       "      <td>2021-06-25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>加强新时代高校网络思想政治教育探究</td>\n",
       "      <td>高歌</td>\n",
       "      <td>学校党建与思想教育</td>\n",
       "      <td>2021-06-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>107.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>试析高校思想政治工作与信息技术的融合</td>\n",
       "      <td>宋丹</td>\n",
       "      <td>学校党建与思想教育</td>\n",
       "      <td>2021-06-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>新媒体时代下老字号品牌传播策略探讨</td>\n",
       "      <td>王宁</td>\n",
       "      <td>商业经济研究</td>\n",
       "      <td>2021-06-21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>121.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>新媒体电商的发展现状及趋势探讨</td>\n",
       "      <td>赵婧宏</td>\n",
       "      <td>商业经济研究</td>\n",
       "      <td>2021-06-21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>348.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>全媒体时代社会主义核心价值观话语权提升路径探析</td>\n",
       "      <td>陈宸</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>新媒体时代高校网络舆情引导机制探析</td>\n",
       "      <td>孙璐</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>新媒体环境下网络广告创意设计研究——评《新媒体时代下的网络广告设计应用》</td>\n",
       "      <td>赵静静</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>短视频教学法助力高校思政课学生满意度的提升——评《思政课多元立体教学模式探索》</td>\n",
       "      <td>秦月; 谭丽琼</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>新媒体视域下的音乐传播研究——评《新媒体音乐编辑与传播》</td>\n",
       "      <td>蔡沅羲</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>新媒体环境下非物质文化遗产的传播与发展——评《文化人类学与非物质文化遗产》</td>\n",
       "      <td>董睿</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>封面人物·许加彪</td>\n",
       "      <td>NaN</td>\n",
       "      <td>新闻爱好者</td>\n",
       "      <td>2021-06-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>新媒体语境下中华民族共同体意识的传播策略  网络首发</td>\n",
       "      <td>沈艾娥</td>\n",
       "      <td>民族学刊</td>\n",
       "      <td>2021-06-15 12:17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>166.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>新媒体环境下对舆论核心要素的再思考</td>\n",
       "      <td>郭小安</td>\n",
       "      <td>南京社会科学</td>\n",
       "      <td>2021-06-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>109.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>德国精英大学计划成员图书馆学术出版素养教育案例研究</td>\n",
       "      <td>吴虚</td>\n",
       "      <td>图书馆工作与研究</td>\n",
       "      <td>2021-06-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>中华优秀传统文化融入高校图书馆阅读推广活动的逻辑路线和实践向度研究</td>\n",
       "      <td>徐荣丽</td>\n",
       "      <td>图书馆</td>\n",
       "      <td>2021-06-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>创新教育管理模式 提高体育教学质量——评《高校体育教学管理改革与模式构建》</td>\n",
       "      <td>杜晓兵</td>\n",
       "      <td>山西财经大学学报</td>\n",
       "      <td>2021-06-11 16:24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>124.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>运用现代设计思维 构建视觉审美元素——评《现代平面设计思维与视觉元素表达》</td>\n",
       "      <td>彭小丹</td>\n",
       "      <td>山西财经大学学报</td>\n",
       "      <td>2021-06-11 16:24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>44.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>新媒体视域下的高校图书馆信息服务——评《高校图书馆读者服务工作拓展与创新》</td>\n",
       "      <td>袁声莉; 徐玲</td>\n",
       "      <td>中国教育学刊</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>144.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>科技赋能：新业态下的教育出版编辑成长策略探析</td>\n",
       "      <td>吴陈滨</td>\n",
       "      <td>中国编辑</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>农林院校畜牧类专业大学计算机线上线下混合教学的探索</td>\n",
       "      <td>刘永福; 苑迎春; 刘丽娟; 王希望; 高玉红</td>\n",
       "      <td>黑龙江畜牧兽医</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>新媒体语境下中国国家形象跨文化传播困境与对策探究</td>\n",
       "      <td>高宝萍; 冯慧</td>\n",
       "      <td>理论导刊</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>南京广电的融媒体创新与升级策略</td>\n",
       "      <td>秘春茜; 李宏刚</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>《湖南工人报》融合发展路径探析</td>\n",
       "      <td>黄义</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>MG动画短视频如何助力抗疫宣传</td>\n",
       "      <td>王皓</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>国家典籍博物馆馆藏甲骨文IP设计开发与推广模式探析</td>\n",
       "      <td>刘玲</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>智能化实践创新 倡导新型教学模式——评《“双一流”建设背景下的传媒教育研究》</td>\n",
       "      <td>李爽</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>31</td>\n",
       "      <td>剖析媒介生态环境 探寻动画变革逻辑——评《基于互联网背景下新媒体动画的发展与创作研究》</td>\n",
       "      <td>韩丹丹</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>32</td>\n",
       "      <td>聚焦政务新媒体的理论落地——评《新媒体视阈下马克思主义大众传播研究》</td>\n",
       "      <td>范经云</td>\n",
       "      <td>传媒</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>33</td>\n",
       "      <td>媒介语料介入认知传播的跨学科研究:谱系与路径</td>\n",
       "      <td>朱婧雯</td>\n",
       "      <td>西南民族大学学报(人文社会科学版)</td>\n",
       "      <td>2021-06-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>34</td>\n",
       "      <td>问题与回应:新媒体时代的政党形象建构</td>\n",
       "      <td>吴阳松</td>\n",
       "      <td>江汉论坛</td>\n",
       "      <td>2021-06-07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>124.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>35</td>\n",
       "      <td>英语学习者在线自我调控及新媒体素养研究——以少数民族地区高中生为例</td>\n",
       "      <td>周琼; 郑春萍; 高梦雅; 王丽丽; 王婧怡</td>\n",
       "      <td>西安外国语大学学报</td>\n",
       "      <td>2021-06-06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>36</td>\n",
       "      <td>风险共担：一个社交媒体环境下新闻发布的变革视角</td>\n",
       "      <td>周庆安; 刘勇亮</td>\n",
       "      <td>新闻与写作</td>\n",
       "      <td>2021-06-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>97.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>37</td>\n",
       "      <td>中西方经典文学中的苦难叙事与女性叙事比较研究——评《新媒体时代的文学经典化》</td>\n",
       "      <td>刘明静</td>\n",
       "      <td>新闻与写作</td>\n",
       "      <td>2021-06-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>34.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>38</td>\n",
       "      <td>新媒体下的广西文化旅游传播——评《旅游景区新媒体营销策略：基于旅游景区与旅游者的双重视角》</td>\n",
       "      <td>马冰琼</td>\n",
       "      <td>新闻与写作</td>\n",
       "      <td>2021-06-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>152.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>39</td>\n",
       "      <td>公共档案馆讲座活动的特点与启示——以美国州立公共档案馆为例</td>\n",
       "      <td>杜慧平; 张书悦</td>\n",
       "      <td>浙江档案</td>\n",
       "      <td>2021-06-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>40</td>\n",
       "      <td>互联网新媒体传播中农村职业培训脱贫致富基本内涵的逻辑生成</td>\n",
       "      <td>杨宗晓; 杨克</td>\n",
       "      <td>农业经济</td>\n",
       "      <td>2021-06-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>113.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>41</td>\n",
       "      <td>企业营销竞争力与消费者选择：社交网络营销的作用机理</td>\n",
       "      <td>周美</td>\n",
       "      <td>商业经济研究</td>\n",
       "      <td>2021-06-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>270.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>42</td>\n",
       "      <td>政务微博“弱”议题舆论引导中的网民情感及其原因分析——基于重要媒体评马保国事件的个案分析</td>\n",
       "      <td>石国良; 王国华</td>\n",
       "      <td>情报杂志</td>\n",
       "      <td>2021-06-01 11:51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>569.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>43</td>\n",
       "      <td>欧洲华文媒体的历史发展与跨文化传播</td>\n",
       "      <td>陈平</td>\n",
       "      <td>华侨华人历史研究</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>44</td>\n",
       "      <td>浅析新媒体环境下图书营销模式</td>\n",
       "      <td>蒋天民</td>\n",
       "      <td>中国出版</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>45</td>\n",
       "      <td>基因、语言与感性：《主播说联播》的“隐形翅膀”</td>\n",
       "      <td>王奋飞; 黄煜</td>\n",
       "      <td>当代电视</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>46</td>\n",
       "      <td>新媒体环境下农民工微信公众号使用行为影响因素分析</td>\n",
       "      <td>常颖; 王晰巍; 窦平安; 陈晓美</td>\n",
       "      <td>情报科学</td>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>47</td>\n",
       "      <td>新媒体与高校思想政治教育研究——评《新媒体视域下高校思想政治教育的解读与重构》</td>\n",
       "      <td>高倩楠</td>\n",
       "      <td>中国测试</td>\n",
       "      <td>2021-05-31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>48</td>\n",
       "      <td>新媒体对高校思政教育的积极影响研究——评《高校网络思政教育平台的构建及其应用研究》</td>\n",
       "      <td>唐海玲</td>\n",
       "      <td>中国测试</td>\n",
       "      <td>2021-05-31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>49</td>\n",
       "      <td>大学生思政教育创新背景下新媒体技术应用研究</td>\n",
       "      <td>林琼</td>\n",
       "      <td>中国测试</td>\n",
       "      <td>2021-05-31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>50</td>\n",
       "      <td>新媒体情境下品牌体验对消费者反应的影响机理研究</td>\n",
       "      <td>许晟; 余明阳; 薛可; 周光</td>\n",
       "      <td>管理评论</td>\n",
       "      <td>2021-05-31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>48.0</td>\n",
       "      <td>下载</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0                                             篇名  \\\n",
       "0            1                          新媒体文学批评在“超真实”世界中的时代变构   \n",
       "1            2                             北京市医院短视频平台传播力现状及展望   \n",
       "2            3                       新媒体视野下常德丝弦的创新与传播研究  网络首发   \n",
       "3            4                            “在场”的言说：新媒体时代影视评论刍议   \n",
       "4            5                              加强新时代高校网络思想政治教育探究   \n",
       "5            6                             试析高校思想政治工作与信息技术的融合   \n",
       "6            7                              新媒体时代下老字号品牌传播策略探讨   \n",
       "7            8                                新媒体电商的发展现状及趋势探讨   \n",
       "8            9                        全媒体时代社会主义核心价值观话语权提升路径探析   \n",
       "9           10                              新媒体时代高校网络舆情引导机制探析   \n",
       "10          11           新媒体环境下网络广告创意设计研究——评《新媒体时代下的网络广告设计应用》   \n",
       "11          12        短视频教学法助力高校思政课学生满意度的提升——评《思政课多元立体教学模式探索》   \n",
       "12          13                   新媒体视域下的音乐传播研究——评《新媒体音乐编辑与传播》   \n",
       "13          14          新媒体环境下非物质文化遗产的传播与发展——评《文化人类学与非物质文化遗产》   \n",
       "14          15                                       封面人物·许加彪   \n",
       "15          16                     新媒体语境下中华民族共同体意识的传播策略  网络首发   \n",
       "16          17                              新媒体环境下对舆论核心要素的再思考   \n",
       "17          18                      德国精英大学计划成员图书馆学术出版素养教育案例研究   \n",
       "18          19              中华优秀传统文化融入高校图书馆阅读推广活动的逻辑路线和实践向度研究   \n",
       "19          20          创新教育管理模式 提高体育教学质量——评《高校体育教学管理改革与模式构建》   \n",
       "20          21          运用现代设计思维 构建视觉审美元素——评《现代平面设计思维与视觉元素表达》   \n",
       "21          22          新媒体视域下的高校图书馆信息服务——评《高校图书馆读者服务工作拓展与创新》   \n",
       "22          23                         科技赋能：新业态下的教育出版编辑成长策略探析   \n",
       "23          24                      农林院校畜牧类专业大学计算机线上线下混合教学的探索   \n",
       "24          25                       新媒体语境下中国国家形象跨文化传播困境与对策探究   \n",
       "25          26                                南京广电的融媒体创新与升级策略   \n",
       "26          27                                《湖南工人报》融合发展路径探析   \n",
       "27          28                                MG动画短视频如何助力抗疫宣传   \n",
       "28          29                      国家典籍博物馆馆藏甲骨文IP设计开发与推广模式探析   \n",
       "29          30         智能化实践创新 倡导新型教学模式——评《“双一流”建设背景下的传媒教育研究》   \n",
       "30          31    剖析媒介生态环境 探寻动画变革逻辑——评《基于互联网背景下新媒体动画的发展与创作研究》   \n",
       "31          32             聚焦政务新媒体的理论落地——评《新媒体视阈下马克思主义大众传播研究》   \n",
       "32          33                         媒介语料介入认知传播的跨学科研究:谱系与路径   \n",
       "33          34                             问题与回应:新媒体时代的政党形象建构   \n",
       "34          35              英语学习者在线自我调控及新媒体素养研究——以少数民族地区高中生为例   \n",
       "35          36                        风险共担：一个社交媒体环境下新闻发布的变革视角   \n",
       "36          37         中西方经典文学中的苦难叙事与女性叙事比较研究——评《新媒体时代的文学经典化》   \n",
       "37          38  新媒体下的广西文化旅游传播——评《旅游景区新媒体营销策略：基于旅游景区与旅游者的双重视角》   \n",
       "38          39                  公共档案馆讲座活动的特点与启示——以美国州立公共档案馆为例   \n",
       "39          40                   互联网新媒体传播中农村职业培训脱贫致富基本内涵的逻辑生成   \n",
       "40          41                      企业营销竞争力与消费者选择：社交网络营销的作用机理   \n",
       "41          42   政务微博“弱”议题舆论引导中的网民情感及其原因分析——基于重要媒体评马保国事件的个案分析   \n",
       "42          43                              欧洲华文媒体的历史发展与跨文化传播   \n",
       "43          44                                 浅析新媒体环境下图书营销模式   \n",
       "44          45                        基因、语言与感性：《主播说联播》的“隐形翅膀”   \n",
       "45          46                       新媒体环境下农民工微信公众号使用行为影响因素分析   \n",
       "46          47        新媒体与高校思想政治教育研究——评《新媒体视域下高校思想政治教育的解读与重构》   \n",
       "47          48      新媒体对高校思政教育的积极影响研究——评《高校网络思政教育平台的构建及其应用研究》   \n",
       "48          49                          大学生思政教育创新背景下新媒体技术应用研究   \n",
       "49          50                        新媒体情境下品牌体验对消费者反应的影响机理研究   \n",
       "\n",
       "                         作者                 刊名              发表时间  被引     下载  \\\n",
       "0                       史丽娜               当代文坛        2021-06-29 NaN    NaN   \n",
       "1          韩扬阳; 李与涵; 高天; 郭蕊               中国医院        2021-06-28 NaN    NaN   \n",
       "2                       田正铁               四川戏剧  2021-06-25 14:42 NaN    NaN   \n",
       "3                  梁振华; 何庆平             中国文艺评论        2021-06-25 NaN    NaN   \n",
       "4                        高歌          学校党建与思想教育        2021-06-23 NaN  107.0   \n",
       "5                        宋丹          学校党建与思想教育        2021-06-23 NaN   26.0   \n",
       "6                        王宁             商业经济研究        2021-06-21 NaN  121.0   \n",
       "7                       赵婧宏             商业经济研究        2021-06-21 NaN  348.0   \n",
       "8                        陈宸              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "9                        孙璐              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "10                      赵静静              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "11                  秦月; 谭丽琼              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "12                      蔡沅羲              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "13                       董睿              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "14                      NaN              新闻爱好者        2021-06-20 NaN    NaN   \n",
       "15                      沈艾娥               民族学刊  2021-06-15 12:17 NaN  166.0   \n",
       "16                      郭小安             南京社会科学        2021-06-15 NaN  109.0   \n",
       "17                       吴虚           图书馆工作与研究        2021-06-15 NaN   15.0   \n",
       "18                      徐荣丽                图书馆        2021-06-15 NaN   12.0   \n",
       "19                      杜晓兵           山西财经大学学报  2021-06-11 16:24 NaN  124.0   \n",
       "20                      彭小丹           山西财经大学学报  2021-06-11 16:24 NaN   44.0   \n",
       "21                  袁声莉; 徐玲             中国教育学刊        2021-06-10 NaN  144.0   \n",
       "22                      吴陈滨               中国编辑        2021-06-10 NaN   20.0   \n",
       "23  刘永福; 苑迎春; 刘丽娟; 王希望; 高玉红            黑龙江畜牧兽医        2021-06-10 NaN    9.0   \n",
       "24                  高宝萍; 冯慧               理论导刊        2021-06-10 NaN    NaN   \n",
       "25                 秘春茜; 李宏刚                 传媒        2021-06-10 NaN    NaN   \n",
       "26                       黄义                 传媒        2021-06-10 NaN    NaN   \n",
       "27                       王皓                 传媒        2021-06-10 NaN    NaN   \n",
       "28                       刘玲                 传媒        2021-06-10 NaN    NaN   \n",
       "29                       李爽                 传媒        2021-06-10 NaN    NaN   \n",
       "30                      韩丹丹                 传媒        2021-06-10 NaN    NaN   \n",
       "31                      范经云                 传媒        2021-06-10 NaN    NaN   \n",
       "32                      朱婧雯  西南民族大学学报(人文社会科学版)        2021-06-10 NaN    NaN   \n",
       "33                      吴阳松               江汉论坛        2021-06-07 NaN  124.0   \n",
       "34   周琼; 郑春萍; 高梦雅; 王丽丽; 王婧怡          西安外国语大学学报        2021-06-06 NaN    5.0   \n",
       "35                 周庆安; 刘勇亮              新闻与写作        2021-06-05 NaN   97.0   \n",
       "36                      刘明静              新闻与写作        2021-06-05 NaN   34.0   \n",
       "37                      马冰琼              新闻与写作        2021-06-05 NaN  152.0   \n",
       "38                 杜慧平; 张书悦               浙江档案        2021-06-05 NaN    4.0   \n",
       "39                  杨宗晓; 杨克               农业经济        2021-06-04 NaN  113.0   \n",
       "40                       周美             商业经济研究        2021-06-03 NaN  270.0   \n",
       "41                 石国良; 王国华               情报杂志  2021-06-01 11:51 NaN  569.0   \n",
       "42                       陈平           华侨华人历史研究        2021-06-01 NaN   64.0   \n",
       "43                      蒋天民               中国出版        2021-06-01 NaN   39.0   \n",
       "44                  王奋飞; 黄煜               当代电视        2021-06-01 NaN   28.0   \n",
       "45        常颖; 王晰巍; 窦平安; 陈晓美               情报科学        2021-06-01 NaN    8.0   \n",
       "46                      高倩楠               中国测试        2021-05-31 NaN   35.0   \n",
       "47                      唐海玲               中国测试        2021-05-31 NaN   56.0   \n",
       "48                       林琼               中国测试        2021-05-31 NaN   49.0   \n",
       "49          许晟; 余明阳; 薛可; 周光               管理评论        2021-05-31 NaN   48.0   \n",
       "\n",
       "    操作  \n",
       "0   下载  \n",
       "1   下载  \n",
       "2   下载  \n",
       "3   下载  \n",
       "4   下载  \n",
       "5   下载  \n",
       "6   下载  \n",
       "7   下载  \n",
       "8   下载  \n",
       "9   下载  \n",
       "10  下载  \n",
       "11  下载  \n",
       "12  下载  \n",
       "13  下载  \n",
       "14  下载  \n",
       "15  下载  \n",
       "16  下载  \n",
       "17  下载  \n",
       "18  下载  \n",
       "19  下载  \n",
       "20  下载  \n",
       "21  下载  \n",
       "22  下载  \n",
       "23  下载  \n",
       "24  下载  \n",
       "25  下载  \n",
       "26  下载  \n",
       "27  下载  \n",
       "28  下载  \n",
       "29  下载  \n",
       "30  下载  \n",
       "31  下载  \n",
       "32  下载  \n",
       "33  下载  \n",
       "34  下载  \n",
       "35  下载  \n",
       "36  下载  \n",
       "37  下载  \n",
       "38  下载  \n",
       "39  下载  \n",
       "40  下载  \n",
       "41  下载  \n",
       "42  下载  \n",
       "43  下载  \n",
       "44  下载  \n",
       "45  下载  \n",
       "46  下载  \n",
       "47  下载  \n",
       "48  下载  \n",
       "49  下载  "
      ]
     },
     "execution_count": 503,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看页面信息\n",
    "element = driver.find_element_by_id('gridTable')\n",
    "page_html = element.get_attribute('innerHTML')\n",
    "page_html\n",
    "pd.read_html(page_html)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 全选文章\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择导出\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择导出格式\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择refworks文件\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/ul/li[8]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.switch_to.window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 批量导出\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[3]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 翻页下载refworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('PageNext').get_attribute('src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input',\n",
       " '/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/i',\n",
       " '/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/a',\n",
       " '/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/ul/li[8]/a']"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "piliang_list = []\n",
    "piliang_list.append(\"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input\")\n",
    "piliang_list.append(\"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/i\")\n",
    "piliang_list.append(\"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/a\")\n",
    "piliang_list.append(\"/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/ul/li[8]/a\")\n",
    "piliang_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "for page in range(0,16):\n",
    "    for i in piliang_list:\n",
    "        element = driver.find_element_by_xpath(i)\n",
    "        element.click()\n",
    "        time.sleep(1)\n",
    "    driver.switch_to.window(driver.window_handles[2])\n",
    "    time.sleep(1)\n",
    "    element = driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[3]/a')\n",
    "    element.click()\n",
    "    time.sleep(1)\n",
    "    driver.close()\n",
    "    driver.switch_to.window(driver.window_handles[1])\n",
    "    time.sleep(1)\n",
    "    element = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/a')\n",
    "    element.click()\n",
    "    driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "    time.sleep(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 翻页自动爬取pdf以及处理验证码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 翻页自动爬取无处理验证码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#翻页爬取pdf文件，无api\n",
    "for u in range(1,3): \n",
    "    for page in range(1,4):\n",
    "        pdf_xpath = '/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{}]/td[2]/a'.format(page)\n",
    "        element = driver.find_element_by_xpath(pdf_xpath)\n",
    "        element.click()\n",
    "        time.sleep(1)\n",
    "        driver.switch_to.window(driver.window_handles[2])\n",
    "        element = driver.find_element_by_xpath('//*[@id=\"pdfDown\"]')\n",
    "        element.click()\n",
    "        time.sleep(1)\n",
    "        driver.close()\n",
    "        driver.switch_to.window(driver.window_handles[1])\n",
    "        time.sleep(1)\n",
    "    driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 百度api运行测试（不使用 精度低）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 505,
   "metadata": {},
   "outputs": [],
   "source": [
    "img()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'9F S'"
      ]
     },
     "execution_count": 506,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baidu_API()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "verifycode_image_url = \"https://kdoc.cnki.net/kdoc/request/ValidateCode.ashx?t=1623825174073\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'24.69e8c0307890e2dcc84877f1415a8b7a.2592000.1627315498.282335-24428692'"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 百度API测试\n",
    "import requests \n",
    "\n",
    "# client_id 为官网获取的AK， client_secret 为官网获取的SK\n",
    "host = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=nc4vvkHyxD4zWMH52r5edGcV&client_secret=yHG63w1ZN6u11F5epku1yu80PvSr2XH6'\n",
    "response = requests.get(host)\n",
    "if response:\n",
    "#     print(response.json())\n",
    "    \n",
    "    access_token = response.json()[\"access_token\"]\n",
    "access_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'words_result': [{'words': '8'}], 'log_id': 1408819316417354266, 'words_result_num': 1}\n"
     ]
    }
   ],
   "source": [
    "request_url = \"https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic\"\n",
    "f = open('screenImg.png', 'rb')\n",
    "img = base64.b64encode(f.read())\n",
    "params = {\n",
    "    \"image\":img\n",
    "}\n",
    "request_url = request_url + \"?access_token=\" + access_token\n",
    "headers = {'content-type': 'application/x-www-form-urlencoded'}\n",
    "response = requests.post(request_url, data=params, headers=headers)\n",
    "response\n",
    "if response:\n",
    "    print (response.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8'"
      ]
     },
     "execution_count": 418,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调用成功，但识别精度不行\n",
    "r=response.json()[\"words_result\"][0][\"words\"]\n",
    "r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 图鉴api运行测试（精度高 可使用）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAALQAAAAyCAAAAADQT64rAAALO0lEQVR4nM1af2wbVx3/3Lvz+Xxxrper47mumwXjhiyKojGFEspGq6pEVdV1WxmojDFGBWiMDo0NVQNKGWwrUjWGGNXEhjShMVBVplBVMKqplE4rXemyKishtJlJPTc1juO6zvlyOZ/vB3/YSfzjzutPiW/+8L33vu/7Pu/7vu/7fe/7QtmwKdwYMukbJJgAFGDfENkNmOcqv6Wa8S5esWACAOaN0nUNFeADZgDAUz3exYx5pZIoG3/4/Cf/Md1++V2K7FXP0bYaLWbRiGYzOdVacxliqMpKzSy5WiDuNOdrqFpEWGXwZiqrGwbhpWDL5QmmbABTN10zQDda2OVOu3I2m1NlidEDsStbO+rG7MEyvb9y8fvXX5/zFb2Vwlw6n2F5QRT4q7I0quzySp7rgLGOSp5nfrBYsqk73ip68aZf1yAEw9fkDd01vTiP+pUtK6yqdkGDrmQbx74dUbPR3TGWhkmfX9GEt9Bq6gwI6JXvXynoS20V5GWrdNhUqFisDao8Qcco9ffKb98XspI6Ghblk77qdW2c77kH49FMrjdp8MFDPuf4xLhgRhtMg6E9gEWj5EE15qmbANwTD+pvUXMsPdtS9HpgbtLS3dprLQuj/NMwDIsw5NOV6enshN4ryFZkNXO0pTy9S21AA2Z75yQvE2G0ZyTDk8b4BKDpRvwFs5XjaJxfUacPc6TPA3zCiE28W9LnndRdOUnPH2VBAf+J51hJ6GNq9b5mootkFRLhkiKnKftXAs7LZ0oSk+EjisUT4z2XE0aDphf4fs7Eno5tY/UT/rZafdBRD1D0E5Y3PfPd70wFRsTfn+ZGo5lIz0dr5FRIyuks6VSMBC8mpPDz+9btbYfPITwYvbl0j3Km/+jtw9GSx9kqXTVt78mFOEG2HtT9DDUVNOrciz2Y07hDAQoXl063v71vQuW1vVGScw+sF79/QspGsiScT0dH/UZweOcTtJPPmlmVZwQhwXRrSW6ccla1C2g7e2hiO17W9IAcuI/467vOcdZgQNWEr5FAcKzn1G6D43InWMPTLEzN8uq2UaNzvEtlMl2TIrffB2C2IQbObcwIJ/qklJ5YlznpqGfXjZg/lvBzZLv+it41NpR6jKNqd/lY3m+kUz3aOi8Ka5V9ohzODuuUp8A3Ca0twKTIZsMyNz6Q1jWSXVGurKO1KU2IMic7AkZe6f/X5YI2aQAkE7qPo+F7pPRcRy4/tIkvV8+l8nmOQOr12iwRdYUArZcEVe8afRc+oLVGkE3VGnZxi5SCmuswBk7cCkuNAHDYi5nYiVTsTIeVFTKC4rIRHc1jZsmfjz8qeAGgpGVPYq2AEaJrxB8MsXRljy3vFU/1HNB8sGNsKP1vZ5XMD2qDQgnr+yfSAow4F8l2jb9XiQT1CrvZ6JZf2Rge9/OyoEw4uzwH85huX2IeC3KWSZuESseNADdmhPqqFpICgA25ZCfJLcWFRwfeGPjbgvXUxroFRRmENhQtk5MFdaJvP2GNtUAjZhCclcaMbFi1RKNcdiRHTU+9GhwcN7JBhe3jlrjdxrz3pZOnGPqrx7qkPcvmg9z08bucuQH7wWRWllhJYYdYjjK1FseT3z2JtIGJXiOS4WVB+cBZlNNGnD3MBcf7eJ2whJrXVqP89SNit0WvEVRMLAMqzqt9nRtmpIZDbEAwMjrHS4ULy1tQ8jgs/uuBgMqTYFxxleMCuiUVGsu/3peQ7mc9mG4HbMohniqxYTHzZJ7pTvuralsb+OZpVyQeY9Kc1rVfK2A5AA+m2xtiUMSwwhOMLOrNQDtajTUu7frpF3tHXjlooh1Vplkj/nSvsfVMt2IEgo6reLa2aB+2GEXVtYAuv4oZlACcP2jXi7YDUl6WmTzHlovOoJ1s2tTB0sDMnxL8Qy7uHcCnSOdYQExIhqVEf8M1zMvcd6+3OuKZGxP9p/hQ3jpSETnnc7C5Uhef9IeHe2UGvCwocedj/rymq7ET1kfPwV5yb+fQvllX0BIykVxeHgpxyG5KNzTTX/JiccgCNuSYRECArM6rwdeYYyihl2f84RQ1ErJYVVC7xlFyBT1VbQEv/lKmp+EDBW/ygUm9diWKi59qNilxwtmWnVxU456eAdAkFdAK3eAlJpsVBtxYZgDPdIrV2NRtNiGywfiNw89PO6qasgG807+I+RkmkN1B2TtYLS2uOmOwT7AggAULFoGaF5dW+ArfTfoz/HO32NTclrzWkXrHHTEAFDeoIIbF+Z//WBMuu9cKpXAkNLk+kFSj/lEtSt5zYGMAILqIeVrIplkK1GMGxyov3b1v+65KA2F0wuXywouVcutw59HuA15Q8B3YIJBI0+SaqT2gMiQjBNI5d8w2ZWt8Lqn5reTuBB8dTYRDmpxx4mQAYOlCsRAP5kJbYNLL1hwBEzwQtJ6tM7zvVH41Lj1gPWzpv7M1ncsguPXJGAGgOvo8mtN0LRjklHy/+8woUH2ZjolueXsy0T3ZIVmKLoSjjpx13mN2x2c2+4BL1lLMvqCIgfXL6nvYFghgaas7050vMS1f7pCO943mEzGRY4jM9zzeOEaBR2YbORNike7Y3zQpdEfCz7K5vF8fOJUP+XVVkxUnr1cfXPiuLAC0AfaeVccig41jUDQA0JykiUfRUvotLjyO6Z18emJgV4SaWfJIHfuzDDhDfWG0n0OG9wu8O2KTtoe2JLk4CQTi4+gwUggIomMcoWzUxuhfWA9xlElflPJDaUHd4W6pd2Iy9Jd5S54bFDSl/9GVsC2dWFV/5MmUJenia5EsHz4dFEe2/sxNXtELFHuJwbGqyhJwlspyWb3j3UpzQfMvxAzKBlBKiwumOPva6cj9gGUczkyuX8+4JXFMsooTkyNUWUU4t2eUM0D+6nQGKnpxR0awhHSAG5nGt3SL9xNVifwIJU/tcbroxXIpIyhByz/co1hqwJjMM05aYwDAE1rE1rIhoB3MqYI/fmvsNod4WNEtbQcAvfINfGTPJv7M4FGHnDTgxQV9IK2yQaS3XDT2GgQAjSLgwWyyex5U0QsvzIAxZRvQE7/aG+M7UmSV4nB+dQzjJf2Amono/Ga+We7KXpsP8X+sKg9y453y2w18Jl3y4M5TQYUVYR2jYZJmvrHQOucDYCp3c4eEMMZvf/MyQQM2ZcIiTSAXWoFvnkTgoHehB+C9f6x71DnGfG4kyKvIioutF5c6MlakAUGJ43MGr485obiqrKl5oq8VDXl4k1gVLRZJ1VY4vwKAuUFTDWv17qXAhddPqaLIfa/SqQaUSc9L9Un+eDcmw473OCfQNvVhTzwV0dVX9Kmb6gDMogUmDVxqg0mb2wQ5l4mhM6/cNnjLIkZn8TY1t0k9QkAYZyTXkJ+uk/ffZShYS1A+MFqG1wRtvnxcyHYyOdVY9TAs4nwudxFaaHXPxjqAvpaXtA9uxvmDo4bAYbwHidg3gnVAG5MzVVSF8VJb+dbpjKW5pi8Dv21B45F7Tq6U07c+5nNK+l7PR50G0CYNVJKh5+KfBco+qhrF4n1k9sjReFdeDE8kNm8DAJQtwGWqDXer6wd6fgTnS7hNYeqN4TzrZ1lNRWztx4FZq/WylqTkgVtu/kqpmXnMj2BTOLfXqlSqqze2A7D12k1yw56Unage9OLgRS/wVFyCJvAJefNXqqvnyaYq5tR4QnG+E1zJG2sTWjiaVsDQODt0hhMtJbUqo/A/aQNM0NX33grmElPJ49AAPGUPVW7wAEWvg/leagP2/rjc8RqpWtMXDpzURLJ640KEXcgffsg4de2FN/pWOvOlll+X/3loZtNX/bjo1LHoBX741FUKrKMb+mJ7TfS+82oBrsnU/wNyvIeX6X+JcqWOVlSKFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=180x50 at 0x29C03520220>"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 图鉴api测试\n",
    "uname = \"gaogaolo\"\n",
    "pwd = \"gaogao1022\"\n",
    "typeid  = \"3\"\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open('screenImg.png', 'rb') as f:    \n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        img = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": img}\n",
    "    time.sleep(1)\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\"\n",
    "\n",
    "\n",
    "# if __name__ == \"__main__\":\n",
    "#     img_path = r\"C:\\Users\\92519\\python数据挖掘\\_week13_\\screenImg.png\"\n",
    "#     result = base64_api(uname='gaogaolo', pwd='', img=img_path, typeid=3)\n",
    "#     print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xwxf'"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base64_api(uname, pwd, img, typeid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理验证码前准备——封装功能函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "#截图\n",
    "def img():\n",
    "    element = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "    element.click()\n",
    "    time.sleep(2)\n",
    "    screenImg = r\"C:\\Users\\92519\\python数据挖掘\\_week13_\\screenImg.png\"\n",
    "    driver.get_screenshot_as_file(screenImg)\n",
    "    location = driver.find_element_by_xpath('//*[@id=\"vImg\"]').location\n",
    "    size = driver.find_element_by_xpath('//*[@id=\"vImg\"]').size\n",
    "    left = location['x']\n",
    "    top = location['y']\n",
    "    right = location['x'] + size['width']\n",
    "    bottom = location['y'] + size['height']\n",
    "    img = Image.open(screenImg).crop((left, top, right, bottom))\n",
    "    img = img.convert('RGBA')\n",
    "    img = img.convert('L')\n",
    "    img = ImageEnhance.Contrast(img)\n",
    "    img = img.enhance(2.0)\n",
    "    img.save(screenImg)\n",
    "    img = Image.open(screenImg)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 判断验证码语句方法1，（不使用）\n",
    "# def img_isTrue():\n",
    "#     element = driver.find_element_by_xpath('//html')\n",
    "#     main_content =element.get_attribute('outerHTML')  \n",
    "#     html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "#     data_list = html.xpath('//*[@id=\"vImg\"]')\n",
    "#     if data_list==[]:\n",
    "#         r = False\n",
    "#     else:\n",
    "#         r = True\n",
    "#     return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 封装百度api，识别精度太低，暂不使用。\n",
    "def baidu_API():\n",
    "    # client_id 为官网获取的AK， client_secret 为官网获取的SK\n",
    "    host = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=rs8gBXsBIilUr0cyU3mgfSH3&client_secret=kwXmD5ifMLb0E3cdndwq3XX2WYTKcPMP'\n",
    "    response = requests.get(host)\n",
    "    if response:\n",
    "    #     print(response.json())\n",
    "\n",
    "        access_token = response.json()[\"access_token\"]\n",
    "        \n",
    "    request_url = \"https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic\"\n",
    "    f = open('screenImg.png', 'rb')\n",
    "    img = base64.b64encode(f.read())\n",
    "    params = {\n",
    "        \"image\":img\n",
    "    }\n",
    "    request_url = request_url + \"?access_token=\" + access_token\n",
    "    headers = {'content-type': 'application/x-www-form-urlencoded'}\n",
    "    response = requests.post(request_url, data=params, headers=headers)\n",
    "    time.sleep(1)\n",
    "    results = response.json()[\"words_result\"][0][\"words\"]\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "#爬取验证码\n",
    "def verifycode():\n",
    "    driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "    element = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "    element.click()\n",
    "    verifycode_image_url = driver.find_element_by_xpath('//*[@id=\"vImg\"]').get_attribute(\"src\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#检测验证码是否存在\n",
    "def img_isTrue():\n",
    "    try:\n",
    "        driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "        return True\n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "#输入验证码\n",
    "def shuru():\n",
    "    key = base64_api(uname, pwd, img, typeid)\n",
    "    print(key)\n",
    "    element = driver.find_element_by_xpath('//*[@id=\"vcode\"]')\n",
    "    element.clear()\n",
    "    element = driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#验证码错误的情况判断\n",
    "def false():\n",
    "    try:\n",
    "        driver.fine_element_by_xpath('//*[@id=\"tips\"]')\n",
    "        return True\n",
    "\n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 切换页面\n",
    "def qiehuan():\n",
    "    all_window=driver.window_handles\n",
    "    new_page = len(all_window)\n",
    "    driver.switch_to.window(driver.window_handles[new_page-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#验证码正确判断：\n",
    "def true():\n",
    "    qiehuan()\n",
    "    driver.close()\n",
    "    driver.switch_to.window(driver.window_handles[2])\n",
    "    time.sleep(1)\n",
    "    qiehuan()\n",
    "    driver.close()\n",
    "    driver.switch_to.window(driver.window_handles[1])\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "#检查点击下载文件数（记得要在selenium浏览器更改下载文件存储的地址）\n",
    "def wenjian():\n",
    "    path = r'C:\\Users\\92519\\python数据挖掘\\_week13_\\pdf'\n",
    "    file = int(len([lists for lists in os.listdir(path) if os.path.isfile(os.path.join(path, lists))]))\n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wenjian2():\n",
    "    path = r'C:\\Users\\92519\\python数据挖掘\\_week13_\\pdf'\n",
    "    file2 = int(len([lists for lists in os.listdir(path) if os.path.isfile(os.path.join(path, lists))]))\n",
    "    return file2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 477,
   "metadata": {},
   "outputs": [],
   "source": [
    "#判断有无下载\n",
    "def wenjian3():\n",
    "    if file2 > file:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动翻页爬取pdf加入处理验证码功能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 505,
   "metadata": {},
   "outputs": [],
   "source": [
    "#返回列表页 报错的时候重新运行时方便用\n",
    "driver.switch_to.window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gt65\n"
     ]
    }
   ],
   "source": [
    "#翻页爬取pdf文件加上封装好的百度api，判断语句没解决\n",
    "for u in range(1,3): \n",
    "    for page in range(1,10):\n",
    "        pdf_xpath = '/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[{}]/td[2]/a'.format(page)\n",
    "        element = driver.find_element_by_xpath(pdf_xpath)\n",
    "        element.click()\n",
    "        time.sleep(1)\n",
    "        driver.switch_to.window(driver.window_handles[2])\n",
    "        element = driver.find_element_by_xpath('//*[@id=\"pdfDown\"]')\n",
    "        element.click()\n",
    "        time.sleep(3)\n",
    "        qiehuan()#切换到最新页面\n",
    "        time.sleep(4)\n",
    "        img_isTrue()#判断是否弹出验证码窗口\n",
    "        res = img_isTrue()\n",
    "        if res is True:\n",
    "            img()#爬取验证码\n",
    "            time.sleep(2)\n",
    "            base64_api(uname, pwd, img, typeid)#调用api识别验证码内容\n",
    "            shuru()#填写识别出来的验证码内容\n",
    "            wenjian()#检测下载文件夹内的文件数量\n",
    "            file = wenjian()\n",
    "            time.sleep(1)\n",
    "            element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "            element.click()\n",
    "            wenjian2()#检测点击下载后文件夹内的文件数量\n",
    "            file2 = wenjian2()\n",
    "            time.sleep(1)\n",
    "            wenjian3()#判断下载文件夹内的文件数量是否变化\n",
    "            res2 = wenjian3()\n",
    "            while res2 == False:\n",
    "                img()#爬取验证码\n",
    "                base64_api(uname, pwd, img, typeid)#调用百度api识别验证码内容\n",
    "                shuru()#填写识别出来的验证码内容\n",
    "                wenjian()#检测下载文件夹内的文件数量\n",
    "                file = wenjian()\n",
    "                element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "                element.click()\n",
    "                wenjian2()\n",
    "                file2 = wenjian2()\n",
    "                wenjian3()#判断下载文件夹内的文件数量是否变化\n",
    "                res2 = wenjian3()\n",
    "                if res2 == True:\n",
    "                    break\n",
    "            true()    \n",
    "        else:\n",
    "            qiehuan()\n",
    "            driver.close()\n",
    "            driver.switch_to.window(driver.window_handles[1])\n",
    "            time.sleep(1)\n",
    "    driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 错误的判断验证码正误的方法汇总，可做逻辑参考"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#错误的方法2\n",
    "#             if res2 is False:\n",
    "#                 verifycode()\n",
    "#                 baidu_API(verifycode_image_url)\n",
    "#                 shuru()\n",
    "#                 wenjian()\n",
    "#                 print(wenjian())\n",
    "#                 element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "#                 element.click()\n",
    "#                 print(wenjian())\n",
    "#                 wenjian2()\n",
    "#                 res3 = wenjian2()\n",
    "#                 if res3 is False:\n",
    "#                     verifycode()\n",
    "#                     baidu_API(verifycode_image_url)\n",
    "#                     shuru()\n",
    "#                     wenjian()\n",
    "#                     print(wenjian())\n",
    "#                     element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "#                     element.click()\n",
    "#                     print(wenjian())\n",
    "#                     wenjian2()\n",
    "#                     res4 = wenjian2()\n",
    "#                     if res4 is False:\n",
    "#                         verifycode()\n",
    "#                         baidu_API(verifycode_image_url)\n",
    "#                         shuru()\n",
    "#                         wenjian()\n",
    "#                         print(wenjian())\n",
    "#                         element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "#                         element.click()\n",
    "#                         print(wenjian())\n",
    "#                         wenjian2()\n",
    "#                         res5 = wenjian2()\n",
    "#                     else:\n",
    "#                         true()\n",
    "#                 else:\n",
    "#                     true()\n",
    "#             else:\n",
    "#                 true()\n",
    "            \n",
    "            \n",
    "# 判断文件数的方法1，有漏洞，验证码即使错误也会自动跳过\n",
    "#             while res2 is False:\n",
    "#                 if res2 is True:\n",
    "#                     break\n",
    "#                 verifycode()#爬取验证码\n",
    "#                 baidu_API(verifycode_image_url)#调用百度api识别验证码内容\n",
    "#                 shuru()#填写识别出来的验证码内容\n",
    "#                 wenjian()#检测下载文件夹内的文件数量\n",
    "#                 element = driver.find_element_by_xpath('/html/body/div/form/dl/dd/button')\n",
    "#                 element.click()\n",
    "#                 wenjian2()#判断下载文件夹内的文件数量是否变化\n",
    "#                 res2 = wenjian2()\n",
    "#             true()\n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "\n",
    "#通过检查页面信息更新来判断是否下载，有漏洞的方法，暂不适用          \n",
    "#             false()\n",
    "#             res2 = false()\n",
    "#             if res2 is True:\n",
    "#                 element = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "#                 element.click()\n",
    "#                 verifycode()\n",
    "#                 baidu_API(verifycode_image_url)\n",
    "#                 shuru()\n",
    "#                 false()\n",
    "#                 res3 = false()\n",
    "#                 if res3 is True:\n",
    "#                     element = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "#                     element.click()\n",
    "#                     verifycode()\n",
    "#                     baidu_API(verifycode_image_url)\n",
    "#                     shuru()\n",
    "#                     false()\n",
    "#                     res4 = false()\n",
    "#                     if res4 is True:\n",
    "#                         element = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "#                         element.click()\n",
    "#                         verifycode()\n",
    "#                         baidu_API(verifycode_image_url)\n",
    "#                         shuru()\n",
    "#                     else:\n",
    "#                         true()\n",
    "#                 else:\n",
    "#                     true()\n",
    "#             else:\n",
    "#                 true()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "229.391px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
